
class Vocabulary(object):
    def __init__(self,token_to_idx=None,add_unk=True,unk_token="<UNK>"):
        '''
        :param token_to_idx: 令牌到数字的映射
        :param add_unk: 是否添加"UNK"令牌
        :param unk_token: 未知词的令牌
        '''
        if token_to_idx is None:
            token_to_idx={}
        self._token_to_idx=token_to_idx

        self._idx_to_token={idx:token
                            for token,idx in self._token_to_idx.items()}
        self._add_unk=add_unk
        self._unk_token=unk_token

        self.unk_index=-1
        if add_unk:
            self.unk_index=self.add_token(unk_token)

    def to_serializable(self):
        return {'token_to_idx':self._token_to_idx,
                'add_unk':self._add_unk,
                'unk_token':self._unk_token}

    @classmethod
    def from_serializable(cls,contents):
        '''实例化序列化词典中的词汇表'''
        return cls(**contents)

    #向词汇表中添加新标记
    def add_token(self,token):
        if token in self._token_to_idx:
            index=self._token_to_idx[token]
        else:
            index=len(self._token_to_idx)
            self._token_to_idx[token]=index
            self._idx_to_token[index]=token
        return index

    def add_many(self,tokens):
        return [self.add_token(token) for token in tokens]

    #检索索引
    def lookup_token(self,token):
        if self.unk_index>=0:
            return self._token_to_idx.get(token,self.unk_index)
        else:
            return self._token_to_idx[token]

    #检索给的索引的标记
    def lookup_index(self,index):
        if index not in self._idx_to_token:
            raise KeyError("the index (%d) is not in the Vocabulary"%index)
        return self._idx_to_token[index]

    def __str__(self):
        return "<Vocabulary(size=%d)>" % len(self)

    def __len__(self):
        return len(self._token_to_idx)

